package com.minispider.util;

import java.net.URL;

import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.PrettyXmlSerializer;

import cn.com.soosoo.robot.http.HttpConstants;
import cn.com.soosoo.robot.http.HttpDoc;
import cn.com.soosoo.robot.http.HttpTool;

public class HtmlDocument
{ 
	private String xPathExp = null;
	
	private String url = null;
	
	private String charSetEncoding = Constants.CHARSET_ENCODING_DEFAULT;
	
	private HtmlCleaner cleaner = null;
	private CleanerProperties props = null;
	private TagNode tNode = null;
	private PrettyXmlSerializer pxs = null;

	private HttpTool ht = new HttpTool();
	
	private static int REDO_REQUEST_TIMES = 10;
	private static int HTTP_REQUEST_TIMEOUT = 20;
	private static String USER_AGENT = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 2.0.50727; CIBA; .NET CLR 1.1.4322)";
	
	public HtmlDocument(String url, String charSetEncoding)
	{		
		this.url = url;
		this.charSetEncoding = charSetEncoding;
		
		ht.setTimeout(HTTP_REQUEST_TIMEOUT);
		ht.setAgentName(USER_AGENT);
		ht.setEnableCookies(true);
	}
	
	private void setHtmlCleanerProps(CleanerProperties props)
	{
		// customize cleaner's behaviour with property setters
		props.setAdvancedXmlEscape(true);
		props.setAllowHtmlInsideAttributes(false);
		props.setAllowMultiWordAttributes(true);
		props.setBooleanAttributeValues("self");
		props.setHyphenReplacementInComment("=");
		props.setIgnoreQuestAndExclam(false);
		props.setNamespacesAware(false);
		props.setOmitComments(false);
		props.setOmitDeprecatedTags(false);
		props.setOmitDoctypeDeclaration(true);
		props.setOmitHtmlEnvelope(false);
		props.setOmitUnknownTags(false);
		props.setOmitXmlDeclaration(false);
		props.setPruneTags("");
		props.setRecognizeUnicodeChars(true);
		props.setTranslateSpecialEntities(true);
		props.setTreatDeprecatedTagsAsContent(false);
		props.setTreatUnknownTagsAsContent(false);
		props.setUseCdataForScriptAndStyle(true);
		props.setUseEmptyElementTags(true);
	}

	public TagNode load(String url, String referer) throws Throwable {
		return getHtmlCleanerTagNode(url, this.charSetEncoding, referer);
	}
	
	private TagNode getHtmlCleanerTagNode(String url, String charSetEncoding, String referer) throws Throwable
	{
		String _url = this.url;
		String _charSetEncoding = this.charSetEncoding;
		
		if (url != null && url.trim().length() != 0)
		{
			_url = url.trim();
		}
		
		if (charSetEncoding != null && charSetEncoding.trim().length() != 0)
		{
			_charSetEncoding = charSetEncoding.trim();
		}
		
		if (_url == null || _url.trim().length() == 0)
		{
			throw new Exception("Class HtmlDocument getHtmlCleanerTagNode(): the param of url is null");
		}
		
		if (_charSetEncoding == null || _charSetEncoding.trim().length() == 0)
		{
			throw new Exception("Class HtmlDocument getHtmlCleanerTagNode(): the param of charSetEncoding is null");
		}
			
		//create an instance of HtmlCleaner
		this.cleaner = new HtmlCleaner();
 
		//take default cleaner properties
		this.props = cleaner.getProperties();
			
		setHtmlCleanerProps(this.props);
	
		int redo = 0;
		do {
			try {
				ht.setReferer(referer);
				
				HttpDoc hd = ht.retrieveDocument(new URL(_url), HttpConstants.GET, "");

				this.tNode = cleaner.clean(new String(hd.getContent(), _charSetEncoding));

				redo = 0;
			} catch (Throwable t) {
				redo++;
				t.printStackTrace();
				Logger.warn("redo http's request now !!! " + " ###### " + redo);
			}
		} while ( redo > 0 && redo < REDO_REQUEST_TIMES);
				
		if (redo != 0) {
			return null;
		}
		
		/*this.cleaner = null;
		this.props = null;*/
		
		return this.tNode;
	}
	
	public StringBuffer getWellFormedHtmlDoc() throws Exception
	{
		return getWellFormedHtmlDoc(null, null);
	}
	
	public StringBuffer getWellFormedHtmlDoc(String url, String charSetEncoding) throws Exception
	{
		String _url = this.url;
		String _charSetEncoding = this.charSetEncoding;
		
		if (url != null && url.trim().length() != 0)
		{
			_url = url.trim();
		}
		
		if (charSetEncoding != null && charSetEncoding.trim().length() != 0)
		{
			_charSetEncoding = charSetEncoding.trim();
		}
		
		if (_url == null || _url.trim().length() == 0)
		{
			throw new Exception("Class HtmlDocument getWellFormedHtmlDoc(): the param of url is null");
		}
		
		if (_charSetEncoding == null || _charSetEncoding.trim().length() == 0)
		{
			throw new Exception("Class HtmlDocument getWellFormedHtmlDoc(): the param of charSetEncoding is null");
		}
		
		//getHtmlCleanerTagNode(_url, _charSetEncoding);
		
		if (this.props == null)
		{
			throw new Exception("Class HtmlDocument getWellFormedHtmlDoc(): the param of this.props is null");
		}
		
		if (this.tNode == null)
		{
			throw new Exception("Class HtmlDocument getWellFormedHtmlDoc(): the param of this.tNode is null");
		}
		
		this.pxs = new PrettyXmlSerializer(this.props);
	    
	    return new StringBuffer(this.pxs.getXmlAsString(this.tNode, _charSetEncoding));
	}
	
	//after called getHtmlCleanerTagNode().
	public Object[] getHtmlDocNodesByXPathExp(String xPathExp) throws Exception
	{
		String _xPathExp = this.xPathExp;
		
		if (xPathExp != null && xPathExp.trim().length() != 0)
		{
			_xPathExp = xPathExp.trim();
		}
		
		if (_xPathExp == null || _xPathExp.trim().length() == 0)
		{
			throw new Exception("Class HtmlDocument getSpecNodesByXPathExp(): the param of xPathExp is null");
		}
		
		//getHtmlCleanerTagNode();

		if (this.tNode == null)
		{
			throw new Exception("Class HtmlDocument getSpecNodesByXPathExp(): the param of this.tNode is null");
		}
		
		Object[] nodes  = this.tNode.evaluateXPath(_xPathExp);
		
		return nodes;
	}
	
	public static void main(String[] args) throws Throwable
	{
	 
		HtmlDocument hd = new HtmlDocument("http://dzh2.mop.com/static/rrlistSubLadyClub/listSubLeft_63_0_0_1.html", Constants.CHARSET_ENCODING_GB2312);
		//HtmlDocument hd = new HtmlDocument("http://bbs.city.tianya.cn/new/tianyacity/ArticlesList_Culture_v3.asp?iditem=39", Utility.CHARSET_ENCODING_GB2312);
		
		hd.load("http://dzh2.mop.com/static/rrlistSubBook/listSubLeft_48_0_0_1.html","");
		Logger.info(hd.getWellFormedHtmlDoc());

		//hd.getHtmlCleanerTagNode();
		
		//System.out.println(hd.getWellFormedHtmlDoc());
		//StringBuffer sb =hd.getWellFormedHtmlDoc();
		//System.out.println(sb);
		
		//Logger.info(hd.getWellFormedHtmlDoc());
		
		/*FileOperation fo = new FileOperation("E:/codes/java/workspace/BBSSpider/www.163.com.txt", "utf-8");
		fo.writeto(hd.getWellFormedHtmlDoc().toString());*/
		
		//Object[] nodes = hd.getHtmlDocNodesByXPathExp("//table[@class=\"tbl\"]/tbody/tr/td[2]/a/text()");
		//Object[] nodes = hd.getHtmlDocNodesByXPathExp("//table[@class=\"tbl\"]/tbody/tr/td[2]/a/@href");
		Object[] nodes = hd.getHtmlDocNodesByXPathExp(
		//"//span[@class=\"st\"]/@id");
	//"//span[@class=\"st\"]/../a[1]/@href");
		//"//span[@class=\"st\"]/../text()");
	"//span[@class=\"st\"]/../text()");

		
		
		//"//div[@id=\"content\"]/table[70]/tbody/tr/td/a[last()]/text()");
				//"//font[@face=\"wingdings\"]/../../td[2]/text()");
				///"//font[@face=\"wingdings\"]/../../td[2]/text()");
			    //"//div[@id=\"mainDiv\"]/table[2]/tbody/tr/td/font[1]/text()");
		//Object[] nodes = hd.getHtmlDocNodesByXPathExp("//table[2]/tbody/tr/td[1]/select/option[last()]/text()");
		
		System.out.println(nodes.length);

	    for (int i = 0; i < nodes.length; i++) {
	    	System.out.println("-----------------------------------");
	        System.out.println(nodes[i].toString().trim()); 
	        
	        if (nodes[i].toString().trim().equals("下一页"))
	        {
		    	System.out.println("&&&&&&&&&&&&&&&&&&&&&&&&&&&");

	        	
	        }
	    }
	    
	    
		/*StringBuffer content = node.g
		
		System.out.println(content);
		
	    FileOperation fo = new FileOperation("E:/codes/java/workspace/test/www.163.com.txt", "utf-8");

		fo.writeto(content.toString());*/
	    
	    /*PrettyXmlSerializer pxs = new PrettyXmlSerializer(props);
	    System.out.println(pxs.getXmlAsString(node, "gb2312"));
	    
	   pxs.writeXmlToFile(node, "E:/codes/java/workspace/test/www.163.com.txt.t", "gb2312");
	    */
	}
}